package com.xxl.job.executor.crawler.job;

import com.xxl.job.core.log.XxlJobLogger;
import com.xxl.job.executor.XxlJobExecutorApplication;
import com.xxl.job.executor.crawler.XxlCrawler;
import com.xxl.job.executor.crawler.annotation.PageFieldSelect;
import com.xxl.job.executor.crawler.annotation.PageSelect;
import com.xxl.job.executor.crawler.conf.XxlCrawlerConf;

import com.xxl.job.executor.crawler.entity.model.BdasCrawlerProduct;
import com.xxl.job.executor.crawler.entity.model.BdasCrawlerProductDetail;
import com.xxl.job.executor.crawler.mapper.BdasCrawlerProductDetailMapper;
import com.xxl.job.executor.crawler.mapper.BdasCrawlerProductMapper;
import com.xxl.job.executor.crawler.parser.PageParser;
import com.xxl.job.executor.crawler.util.JedisUtil;
import com.xxl.job.executor.crawler.util.ListUtiil;
import com.xxl.job.executor.crawler.util.UUIDUtil;

import lombok.Data;
import org.jsoup.nodes.Document;
import org.jsoup.nodes.Element;
import org.junit.Test;
import org.junit.runner.RunWith;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.boot.test.context.SpringBootTest;
import org.springframework.data.repository.query.Param;
import org.springframework.stereotype.Service;
import org.springframework.test.context.junit4.SpringRunner;
import redis.clients.jedis.Jedis;

import java.math.BigDecimal;
import java.sql.Timestamp;
import java.util.*;

/*
 *  lcx 爬取所有商品的详细信息
 * */

@Service
@RunWith(SpringRunner.class)
@SpringBootTest(classes = XxlJobExecutorApplication.class, webEnvironment = SpringBootTest.WebEnvironment.RANDOM_PORT)
public class AllProductDetail {
    @PageSelect(cssQuery = "body")
    @Data
    public static class PageVo {
        //商品名称
        @PageFieldSelect(cssQuery = "#productTrackingParams", selectType = XxlCrawlerConf.SelectType.ATTR, selectVal = "data-title")
        private String product_detail_name;

        //商品名称 用于 缺货的商品
        @PageFieldSelect(cssQuery = ".product-title > h6", selectType = XxlCrawlerConf.SelectType.HTML)
        private String product_detail_name_two;

        //商品的超链接
       /* @PageFieldSelect(cssQuery = "", selectType = XxlCrawlerConf.SelectType.ATTR, selectVal = "href")
        private List<String> product_href;*/

        //图片链接
        @PageFieldSelect(cssQuery = ".zoom-enabled > img", selectType = XxlCrawlerConf.SelectType.ATTR, selectVal = "src")
        private String product_img_href;

        //图片链接 缺货的商品
        @PageFieldSelect(cssQuery = ".vip-outofstock-item-img-container > img", selectType = XxlCrawlerConf.SelectType.ATTR, selectVal = "src")
        private String product_img_href_two;

        /* SPECIFICATIONS 键*/
        @PageFieldSelect(cssQuery = "#specs-full dt", selectType = XxlCrawlerConf.SelectType.HTML)
        private List<String> product_grouped_list_key;

        /* SPECIFICATIONS 值*/
        @PageFieldSelect(cssQuery = "#specs-full dd", selectType = XxlCrawlerConf.SelectType.HTML)
        private List<String> product_grouped_list_value;

        @PageFieldSelect(cssQuery = "#specs-short dt", selectType = XxlCrawlerConf.SelectType.HTML)
        private List<String> product_groupedOther_list_key;

        @PageFieldSelect(cssQuery = "#specs-short dd", selectType = XxlCrawlerConf.SelectType.HTML)
        private List<String> product_groupedOther_list_value;

        /* 商品现价格*/
        @PageFieldSelect(cssQuery = "h3.price.is", selectType = XxlCrawlerConf.SelectType.HTML)
        private String now_price;

        /* 商品打折前价格*/
        @PageFieldSelect(cssQuery = ".price-container span.was", selectType = XxlCrawlerConf.SelectType.HTML)
        private String old_price;

        /* 是否上新 0为否*/
        @PageFieldSelect(cssQuery = "dd.unit-condition", selectType = XxlCrawlerConf.SelectType.HTML)
        private String is_condition;

        /* 是否官方仓 0为否*/
        @PageFieldSelect(cssQuery = "#Fullfilled-tooltip > li > span", selectType = XxlCrawlerConf.SelectType.HTML)
        private String is_souq;

        /*品牌*/
        @PageFieldSelect(cssQuery = "#productTrackingParams", selectType = XxlCrawlerConf.SelectType.ATTR, selectVal = "data-brand-name")
        private String brand;

        /*商品类型*/
        @PageFieldSelect(cssQuery = "#productTrackingParams", selectType = XxlCrawlerConf.SelectType.ATTR, selectVal = "data-category-name")
        private String product_type;

        /*用于缺货的商品类型*/
        @PageFieldSelect(cssQuery = ".product-title > span > a", selectType = XxlCrawlerConf.SelectType.HTML)
        private List<String> product_title_two;

        /* 库存量 为空代表收集不到*/
        @PageFieldSelect(cssQuery = ".unit-labels > b > span", selectType = XxlCrawlerConf.SelectType.HTML)
        private String inventory;

        /* 来自于那家店铺*/
        @PageFieldSelect(cssQuery = "a > b", selectType = XxlCrawlerConf.SelectType.HTML)
        private String sold_by;

        /* 跟卖数量*/
        @PageFieldSelect(cssQuery = ".other-sellers > .show-for-medium > a > font > font", selectType = XxlCrawlerConf.SelectType.HTML)
        private String more_offers_num;

        /* 更卖商品链接集合*/
        @PageFieldSelect(cssQuery = "a.show-for-medium.bold-text", selectType = XxlCrawlerConf.SelectType.ATTR, selectVal = "href")
        private String more_offers_product_href;

        /* 官方推荐组合商品链接集合*/
        @PageFieldSelect(cssQuery = ".slick-active a.itemLink", selectType = XxlCrawlerConf.SelectType.ATTR, selectVal = "href")
        private List<String> complete_product_href;

        /* 官方推荐组合商品enid 集合*/
        @PageFieldSelect(cssQuery = ".slick-active a.itemLink", selectType = XxlCrawlerConf.SelectType.ATTR, selectVal = "data-enid")
        private List<String> complete_product_enid;

        /* 官方推荐好卖商品链接集合*/
        @PageFieldSelect(cssQuery = ".PopularItems a.itemLink", selectType = XxlCrawlerConf.SelectType.ATTR, selectVal = "href")
        private List<String> recommend_product_href;

        /* 官方推荐好卖商品enid 集合*/
        @PageFieldSelect(cssQuery = ".PopularItems a.itemLink", selectType = XxlCrawlerConf.SelectType.ATTR, selectVal = "data-enid")
        private List<String> recommend_product_enid;


        /* 官方赞助商品链接集合*/
        @PageFieldSelect(cssQuery = ".ad-sponsored-item a.itemLink", selectType = XxlCrawlerConf.SelectType.ATTR, selectVal = "href")
        private List<String> sponsorship_product_href;

        /* 官方赞助商品链接集合 enid*/
        @PageFieldSelect(cssQuery = ".ad-sponsored-item a.itemLink", selectType = XxlCrawlerConf.SelectType.ATTR, selectVal = "data-enid")
        private List<String> sponsorship_product_enid;

        /* 评论星星1-5*/
        @PageFieldSelect(cssQuery = "#progress_bar > span.progress-meter", selectType = XxlCrawlerConf.SelectType.ATTR, selectVal = "style")
        private List<String> stars1_5;

        /* 评论星星5*/
        @PageFieldSelect(cssQuery = ".rate-avg > span.starRating_5", selectType = XxlCrawlerConf.SelectType.HTML)
        private String stars_5;

        /* 评论星星4*/
        @PageFieldSelect(cssQuery = ".rate-avg > span.starRating_4", selectType = XxlCrawlerConf.SelectType.HTML)
        private String stars_4;

        /* 评论星星3*/
        @PageFieldSelect(cssQuery = ".rate-avg > span.starRating_3", selectType = XxlCrawlerConf.SelectType.HTML)
        private String stars_3;

        /* 评论星星2*/
        @PageFieldSelect(cssQuery = ".rate-avg > span.starRating_2", selectType = XxlCrawlerConf.SelectType.HTML)
        private String stars_2;

        /* 评论星星1*/
        @PageFieldSelect(cssQuery = ".rate-avg > span.starRating_1", selectType = XxlCrawlerConf.SelectType.HTML)
        private String stars_1;

        /* 评论星星1*/
        @PageFieldSelect(cssQuery = ".avg > strong", selectType = XxlCrawlerConf.SelectType.HTML)
        private String stars;

        /* 满意度*/
        @PageFieldSelect(cssQuery = ".rate-product-section > h6 > span > font > font", selectType = XxlCrawlerConf.SelectType.HTML)
        private String satisfaction;

        /* ean*/
        @PageFieldSelect(cssQuery = "#productTrackingParams", selectType = XxlCrawlerConf.SelectType.ATTR, selectVal = "data-ean")
        private String product_ean;


    }

    @Autowired
    protected BdasCrawlerProductMapper bdasCrawlerProductMapper;
    @Autowired
    protected BdasCrawlerProductDetailMapper bdasCrawlerProductDetailMapper;

    /**
     * @Author lcx
     * @Date 2018/11/29 16:01
     * 爬虫逻辑部分
     *
     *              *  此处根据传进的参数，判断爬取哪个商品，
     *
     */
    @Test
    public void start_crawler(String param) {

        Map headerMap = new HashMap();
        Map cookieMap = new HashMap();

        cookieMap.put("BUYER_CITY_SELECTED", "80");

        headerMap.put("Cache-Control", "max-age=0");

        Map paramMap = new HashMap();

        paramMap.put("type","2");

        final List<BdasCrawlerProductDetail> list = new ArrayList<>();

        final Timestamp begind_time = new Timestamp(System.currentTimeMillis());

        final int max_v = bdasCrawlerProductDetailMapper.findByMaxVersion();


        //redis有数据则执行redis
        if (JedisUtil.selectDetailUrlByRedis().size() != 0) {

            XxlJobLogger.log("爬取失败队列");

            XxlJobLogger.log("失败队列数：",JedisUtil.selectDetailUrlByRedis());

            List listJedis = JedisUtil.selectDetailUrlByRedis();

            // 给url数组赋长度：集合长度
            final String[] link = new String[listJedis.size()];
            Jedis jedis = JedisUtil.getJedis();

            // 循环遍历集合中的url添加到数组中
            for (int i = 0; i < listJedis.size(); i++) {
                link[i] = jedis.lpop("detailUrl");
            }

            XxlJobLogger.log("初始化任务数量:" + listJedis.size());

            crawlerUrl(link,cookieMap,headerMap,paramMap,bdasCrawlerProductMapper,bdasCrawlerProductDetailMapper,list,begind_time,max_v,param);

        }else {

            XxlJobLogger.log("爬取数据库链接");

            //构建爬取池
            //List<BdasCrawlerProductEntity> bdasCrawlerProductEntityList = bdasCrawlerProductDao.findAll();
            List<BdasCrawlerProduct> bdasCrawlerProductEntityList = new ArrayList<>();
            if (param.equals("Beats")) {
                bdasCrawlerProductEntityList = bdasCrawlerProductMapper.findBeatsByType();
            } else if (param.equals("Bose")) {
                bdasCrawlerProductEntityList = bdasCrawlerProductMapper.findBoseByType();
            } else if(param.equals("Casio")) {
                bdasCrawlerProductEntityList = bdasCrawlerProductMapper.findCasioByType();
            } else if(param.equals("Diesel")) {
                bdasCrawlerProductEntityList = bdasCrawlerProductMapper.findDieselByType();
            } else if(param.equals("Tissot")) {
                bdasCrawlerProductEntityList = bdasCrawlerProductMapper.findTissotByType();
            }

            XxlJobLogger.log("初始化任务数量:" + bdasCrawlerProductEntityList.size());

            final String[] link = new String[bdasCrawlerProductEntityList.size()];


            for (int j = 0; j < link.length; j++) {
                link[j] = bdasCrawlerProductEntityList.get(j).getProductHref();
            }

            crawlerUrl(link,cookieMap,headerMap,paramMap,bdasCrawlerProductMapper,bdasCrawlerProductDetailMapper,list,begind_time,max_v,param);
        }

    }

    /**
     *  封装爬虫
     * @param link
     * @param cookieMap
     * @param headerMap
     * @param paramMap
     * @param
     * @param list
     * @param begind_time
     * @param max_v
     */
    public static void crawlerUrl(String[] link, Map cookieMap, Map headerMap,Map paramMap,
                                  BdasCrawlerProductMapper bdasCrawlerProductMapper,
                                  BdasCrawlerProductDetailMapper bdasCrawlerProductDetailMapper,
                                  final List<BdasCrawlerProductDetail> list,
                                  final Timestamp begind_time,
                                  final int max_v,
                                  String param) {
        XxlCrawler crawler = new XxlCrawler.Builder()
                .setUrls(link)
                .setThreadCount(6).setPauseMillis(1000)
                .setAllowSpread(false)
                //.setProxyMaker(proxyMaker)
                .setParamMap(paramMap)
                .setCookieMap(cookieMap)
                .setHeaderMap(headerMap)
                .setPageParser(
                        new PageParser<PageVo>() {

                            @Override
                            public boolean parse(Document html, Element pageVoElement, PageVo pageVo) {
                                String theUrl = html.baseUri();

                                try {
                                String item_id = theUrl.substring(theUrl.lastIndexOf("-")+1, theUrl.indexOf("/i/"));

                                BdasCrawlerProductDetail bdasCrawlerProductDetailEntity = new BdasCrawlerProductDetail();

                                //商品名称
                                bdasCrawlerProductDetailEntity.setProductDetailName(pageVo.getProduct_detail_name());

                                //商品ean
                                bdasCrawlerProductDetailEntity.setProductEan(pageVo.getProduct_ean());

                                //商品图片链接
                                bdasCrawlerProductDetailEntity.setProductImgHref(pageVo.getProduct_img_href());

                                //商品链接
                                bdasCrawlerProductDetailEntity.setProductHref(html.baseUri());


                                try{
                                //商品发布日期
                                if(pageVo.getProduct_grouped_list_key()!=null) {

                                    bdasCrawlerProductDetailEntity.setReleaseDate(ListUtiil.ListToValue(pageVo.getProduct_grouped_list_key(),
                                            pageVo.getProduct_grouped_list_value(), XxlCrawlerConf.Key_Code.Release_Date.getValue()));
                                }else {

                                    bdasCrawlerProductDetailEntity.setReleaseDate(ListUtiil.ListToValue(pageVo.getProduct_groupedOther_list_key(),
                                            pageVo.getProduct_groupedOther_list_value(), XxlCrawlerConf.Key_Code.Release_Date.getValue()));
                                }

                                //品牌
                                bdasCrawlerProductDetailEntity.setBrand(pageVo.getBrand());

                                //类型
                                bdasCrawlerProductDetailEntity.setProductType(pageVo.getProduct_type());

                                //商品现价格
                                bdasCrawlerProductDetailEntity.setNowPrice(new BigDecimal(ListUtiil.StringToStirngAsReplace(
                                        pageVo.getNow_price(), 10, 24,
                                        XxlCrawlerConf.Replace_Code.REPLACE_CODE_ISNUM.getValue())));

                                //商品打折前价格
                                bdasCrawlerProductDetailEntity.setOldPrice(new BigDecimal(ListUtiil.StringToStirngAsReplace(
                                        pageVo.getOld_price(), 0, 0,
                                        XxlCrawlerConf.Replace_Code.REPLACE_CODE_ISNUM.getValue())));
                                }catch (Exception e){
                                    XxlJobLogger.log("出错的连接是："+theUrl);
                                }
                                //是否上新 0为否
                                bdasCrawlerProductDetailEntity.setIsCondition(ListUtiil.IsTrue(pageVo.getIs_condition()));

                                //是否官方仓 0为否
                                bdasCrawlerProductDetailEntity.setIsSouq(ListUtiil.IsTrue(pageVo.getIs_souq()));

                                //来自于那家店铺
                                bdasCrawlerProductDetailEntity.setSoldBy(pageVo.getSold_by());

                                //跟卖数量
                                bdasCrawlerProductDetailEntity.setMoreOffersNum(pageVo.getMore_offers_num());

                                //更卖商品链接集合
                                bdasCrawlerProductDetailEntity.setMoreOffersProductHref(pageVo.getMore_offers_product_href());

                                //官方推荐组合商品链接集合
                                bdasCrawlerProductDetailEntity.setCompleteProductHref(ListUtiil.ListToString(pageVo.getComplete_product_href(), ","));

                                //评论星星5
                                bdasCrawlerProductDetailEntity.setStars5(ListUtiil.StringToStirngAsReplace(
                                        pageVo.getStars_5(), 0, 0,
                                        XxlCrawlerConf.Replace_Code.REPLACE_CODE_ISNUM.getValue()));

                                //评论星星4
                                bdasCrawlerProductDetailEntity.setStars4(ListUtiil.StringToStirngAsReplace(
                                        pageVo.getStars_4(), 0, 0,
                                        XxlCrawlerConf.Replace_Code.REPLACE_CODE_ISNUM.getValue()));

                                //评论星星3
                                bdasCrawlerProductDetailEntity.setStars3(ListUtiil.StringToStirngAsReplace(
                                        pageVo.getStars_3(), 0, 0,
                                        XxlCrawlerConf.Replace_Code.REPLACE_CODE_ISNUM.getValue()));

                                //评论星星2
                                bdasCrawlerProductDetailEntity.setStars2(ListUtiil.StringToStirngAsReplace(
                                        pageVo.getStars_2(), 0, 0,
                                        XxlCrawlerConf.Replace_Code.REPLACE_CODE_ISNUM.getValue()));

                                //评论星星1
                                bdasCrawlerProductDetailEntity.setStars1(ListUtiil.StringToStirngAsReplace(
                                        pageVo.getStars_1(), 0, 0,
                                        XxlCrawlerConf.Replace_Code.REPLACE_CODE_ISNUM.getValue()));

                                //评论星星
                                bdasCrawlerProductDetailEntity.setStars(pageVo.getStars());

                                //时间戳开始时间
                                bdasCrawlerProductDetailEntity.setCreateDate(new Timestamp(System.currentTimeMillis()));

                                //字符串时间
                                bdasCrawlerProductDetailEntity.setCreateDateAsDay(ListUtiil.getTimeDateFormat("yyyy-MM-dd"));

                                bdasCrawlerProductDetailEntity.setCreateDateAsMonth(ListUtiil.getTimeDateFormat("yyyy-MM"));

                                bdasCrawlerProductDetailEntity.setCreateDateAsYears(ListUtiil.getTimeDateFormat("yyyy"));

                                bdasCrawlerProductDetailEntity.setGenerateDate(begind_time);

                                bdasCrawlerProductDetailEntity.setProductIdItem(item_id);

                                int max = max_v + 1;
                                bdasCrawlerProductDetailEntity.setVersion(max);
                                bdasCrawlerProductDetailEntity.setProductDetailId(UUIDUtil.getId());


                                //如果是 缺货状态 替换名称 库存数量为0
                                if (pageVo.getProduct_detail_name() == null) {
                                    bdasCrawlerProductDetailEntity.setProductDetailName(pageVo.getProduct_detail_name_two());
                                    bdasCrawlerProductDetailEntity.setInventory("0");
                                    bdasCrawlerProductDetailEntity.setProductImgHref(pageVo.getProduct_img_href_two());

                                    if (pageVo.getProduct_title_two().size() >= 1) {
                                        bdasCrawlerProductDetailEntity.setBrand(pageVo.getProduct_title_two().get(0));
                                        bdasCrawlerProductDetailEntity.setProductType(pageVo.getProduct_title_two().get(1));
                                    }
                                }

                                if (bdasCrawlerProductDetailEntity.getProductEan() != null) {
                                    list.add(bdasCrawlerProductDetailEntity);
                                }
                                } catch (Exception e) {
                                    XxlJobLogger.log("爬虫出错的连接为："+theUrl);
                                }
                                return true;
                            }
                        }
                )
                .build();

        XxlJobLogger.log("任务构建完毕，开始执行！");
        crawler.start(true);
        XxlJobLogger.log("执行完毕！");
        XxlJobLogger.log("执行成功数量数量:" + list.size());
        XxlJobLogger.log("开始持久化！");

        //int i  = 1;
        /*for (BdasCrawlerProductDetail tem : list) {
            bdasCrawlerProductDetailMapper.insertSelective(tem);
            //XxlJobLogger.log("正在持久化"+ i++ +"条");
            i++;
        }*/
        int insert = bdasCrawlerProductDetailMapper.insert(list);

        XxlJobLogger.log("持久化成功"+ insert +"条");

        bdasCrawlerProductMapper.updateStatusByType(param);

        XxlJobLogger.log("持久化结束！");

        if (!crawler.isRunState()) {
            XxlJobLogger.log("-----------------这个类别已结束-------------------------");
        }
    }
}


